import scrapy

from ..items import BiqugeItem


class CrawlNovelSpider(scrapy.Spider):
    name = "crawl_novel"
    # allowed_domains = ["baidu.com"]
    start_urls = ["https://www.ydxrf.com/html/51434/"]
    base_url="https://www.ydxrf.com/"
    def parse(self, response):
        book_name = response.css(".infobar h1::text").get()
        content_list = response.css("#list dd a ::attr(href)").getall()
        for content in content_list:
            content_url=self.base_url+content
            yield scrapy.Request(
                url=content_url,
                callback=self.parse_content,
                meta={'book_name':book_name}
            )

    def parse_content(self,response):
        title = response.css(".bookname h1::text").get()
        novel_content=response.css("#htmlContent p::text").getall()
        result=[]
        for i in novel_content:
            a=str(i).strip()
            result.append(a)
        novel_content='\n'.join(result)
        yield BiqugeItem(title=title,novel_content=novel_content,book_name=response.meta['book_name'])